#!/usr/bin/env python

# 1) Collecting the system infomation of local machine and the virtual machines on it
# 2) Send the info to the ivic portal REST interface by the tool 'curl' 

import os
import libvirt
import sys, libxml2, time
from xml.dom import minidom
from ivic.core import Config, project_path, Logging
from ivic.core.Utils import check_pid_file

import subprocess
import re
config_monitor_path = project_path("/etc/ivic/monitor.yaml")
config_daemon_path = project_path("/etc/ivic/daemon.yaml")
pid_file = project_path("/var/run/ivic/monitor.pid")
log_file = project_path("/var/log/ivic/monitor.log")

logger = Logging.get_logger(__name__, filename = log_file, default = True)  

def exe_command(command):
  pipe=os.popen(command,"r")
  result=pipe.readlines()
  pipe.close()
  return result

def get_pm_cpu_info():
  cpu_type=''
  cpu_num=0
  cpu_usage=''
  cpu_info=[]
  try:
    cputype_line=exe_command("cat /proc/cpuinfo | grep -m 1 'model name'")
    cpu_type=cputype_line[0].split(":")[1][1:].strip()
    cpunum_lines=exe_command("cat /proc/cpuinfo | grep 'processor'")
    cpu_num=int(cpunum_lines[len(cpunum_lines)-1].split()[2])+1
    cputime_line=exe_command("head -n 1 /proc/stat")[0]
    t1=int(cputime_line.split()[1])
    t2=int(cputime_line.split()[2])
    t3=int(cputime_line.split()[3])
    t4=int(cputime_line.split()[4])
    time.sleep(5)
    cputime_line=exe_command("head -n 1 /proc/stat")[0]
    t1=int(cputime_line.split()[1])-t1
    t2=int(cputime_line.split()[2])-t2
    t3=int(cputime_line.split()[3])-t3
    t4=int(cputime_line.split()[4])-t4
    cpu_usage=float(t1+t2+t3)/(t1+t2+t3+t4) * 0.2
    cpu_info.append(cpu_type)
    cpu_info.append(str(cpu_num))
    cpu_info.append(str(cpu_usage))
    return cpu_info
  except:
    print "Error in get_cpu_info()"
    return ["NA","NA","NA"]

def get_pm_default_interface():
  try:
    routes=exe_command("/sbin/route -n | sed '1,2d' | grep 'UG'")
    return routes[0].split()[7]
  except:
    print "Error in get_default_interface"
  return "eth0"

def get_pm_disk_info():
  disk_device=''
  disk_total=''
  disk_free=''
  disk_info=[]
  try:
    disklines=exe_command("df -P | grep '^\/dev\/'")
    for line in disklines:  
      disk_device+=line.split()[0]+" "
      disk_total+=line.split()[1]+" "
      disk_free+=line.split()[3]+" "
    disk_info.append(disk_device)
    disk_info.append(disk_total)
    disk_info.append(disk_free)
    return disk_info
  except:
    print "Error in get_disk_info()"
    return ["NA","NA","NA"]

def get_pm_ip(ifname = "eth0"):
  try:
    ip_line=exe_command("/sbin/ifconfig " + ifname + " | grep 'inet addr:'")
    return [ip_line[0].split(":")[1].split()[0]]
  except:
    print "Error in get_ip()"
    return ["127.0.0.1"]

def get_pm_mem_info():
  mem_info=[]
  try:
    meminfo_lines=exe_command("cat /proc/meminfo")
    mem_info.append(str(int(meminfo_lines[0].split()[1])))
    mem_info.append(str(int(meminfo_lines[1].split()[1])+int(meminfo_lines[2].split()[1])+int(meminfo_lines[3].split()[1])))
    return mem_info
  except:
    print "Error in get_mem_info()"
    return ["NA","NA"]

def get_pm_network_info():
  #net_info=[net_ifname,net_tx,net_rx]
  network_info = []
  net_ifname = ""
  net_tx = ""
  net_rx = ""
  try:
    eth_list = exe_command("/sbin/ifconfig | grep ^eth")
    for eth in eth_list:
      cur_eth_name = eth.split()[0]
      net_ifname += cur_eth_name+" "
      command="/sbin/ifconfig " + cur_eth_name + "| grep 'TX bytes:'"
      tx_line=exe_command(command)[0]
      net_tx += tx_line.split("(")[2].split()[0]+" "
      net_rx += tx_line.split("(")[1].split()[0]+" "
    network_info.append(net_ifname)
    network_info.append(net_tx)
    network_info.append(net_rx)
    return network_info
  except:
    print "Error in get_network_info()"
    return ["NA","NA","NA"]

def get_domain_device_path_from_xml(vm_xml_doc, device_name):
    doc = libxml2.readDoc(vm_xml_doc, None, None, libxml2.XML_PARSE_NOENT)
    ctxt = doc.xpathNewContext()
    res = ctxt.xpathEval("/domain/devices/%s/target[@dev]" % device_name)
    if type(res) != type([]) or len(res) == 0:
       doc.free()
       return None
    disk_paths = []
    for i in xrange(len(res)):
      disk_paths.append(res[i].properties.content)
    doc.free()
    return disk_paths

def get_domain_disk_info(conn, domain):
  disk_paths = get_domain_device_path_from_xml(domain.XMLDesc(0), "disk")
  disk_read = 0
  disk_write = 0
  if disk_paths == None:
    return ["NA", "NA", "NA"]
  for disk_path in disk_paths:
    try:
      disk_read += float(domain.blockStats(disk_path)[1])
      disk_write += float(domain.blockStats(disk_path)[3])
    except:
      pass
  return [":".join(disk_paths), str(disk_read), str(disk_write)]

def get_domain_vif_info(conn, domain):
  vif_paths = get_domain_device_path_from_xml(domain.XMLDesc(0), "interface")
  vif_tx = 0
  vif_rx = 0
  if vif_paths == None:
    return ["NA", "NA", "NA"]
  for vif_path in vif_paths:
    vif_tx += float(domain.interfaceStats(vif_path)[0])
    vif_rx += float(domain.interfaceStats(vif_path)[4])
  return [":".join(vif_paths), str(vif_tx), str(vif_rx)]

def get_domain_vnc_port(domain_id, uuid):
  try:
    #vncdisplay = minidom.parseString(domain.XMLDesc(0)).getElementsByTagName('graphics')[0].getAttribue('port')
    vncdisplay = exe_command("virsh vncdisplay " + domain_id)
    os.system("vncsnapshot -allowblank %s /var/lib/ivic/vmc/images/%s.snapshot.jpg" % (vncdisplay[0].strip(), uuid))
    return vncdisplay[0].split(":")[1].strip()
    #return str(vncdisplay)[3]
    #if vncdisplay == ['\n']:
    #  print 'True'
    #  return 'NA'
    #else:
    #  return vncdisplay[0].split(":")[1].strip()
  except:
    print "Error in get vnc port of domain" + domain_id
    return "NA"

def get_domains_info(conn, hypervisor_type, domains_ID):
  domains_info = []
  for id in domains_ID:
    domain_info = []
    domain = conn.lookupByID(id)
    domain_info.append(domain.UUIDString()) # vm_uuid
    domain_info.append(domain.name()) # vm_name
    domain_info.append("running") # vm_state     
    domain_info.append(domain.info()[1]) # vm_memtotal
    domain_info.append(domain.info()[1]-domain.info()[2]) # vm_memfree
    domain_info.append(float(domain.info()[4])/1000000000) # vm_cputime(seconds) 
    domain_info += get_domain_disk_info(conn, domain)
    domain_info += get_domain_vif_info(conn, domain)
    domain_info += [get_domain_vnc_port(str(id), domain.UUIDString())]
    domains_info.append(domain_info)
  return domains_info

def do_main(argv):
  argv =  argv[1:]
  client_key =  "/root/123.pem"
  server_cert = "/root/ca.crt"
  send_url = "https://portal.ivic.org.cn/vmcs/api"
  capability = '' #default capability = empty string
  port = '8080'
  arg_names = [     "address",      "status",    "cpu_type",     "cpu_num",   "cpu_usage",
                  "mem_total",    "mem_free", "disk_device",  "disk_total",   "disk_free",
                 "net_ifname",      "net_tx",      "net_rx",  "capability",        "port",
                    "vm_uuid",     "vm_name",    "vm_state",  "vmem_total",   "vmem_free",
                 "vcpu_usage", "vdisk_names",  "vdisk_read", "vdisk_write",   "vif_names",
                     "vif_tx",      "vif_rx",    "vnc_port", "all_pair_ping"]
  # In fact, "vcpu_usage" corresponds to the column 'runtime1' in portal db vmis table.
  # vdisk_name, vif_name are used for show the collectd info in portal.
  arg_values = []

  # analyse the file monitor.yaml
  try:
    config_monitor = Config.load(config_monitor_path)
    client_key = config_monitor['client_key']
    server_cert = config_monitor['server_cert']
    send_url = config_monitor['monitor_send_url']

    # config item 'capability' is list, convert it to a space separated string
    capability = ' '.join(config_monitor['capabilities'])
  except Exception, e:
    print e
    print "Could not find config in %s, use default setting." % config_monitor_path

  # analyse the file daemon.yaml to get 'port'
  try:
    port = Config.load(config_daemon_path)['listen_port']
  except Exception, e:
    print e
    print "Could not find config in %s, use default setting." % config_daemon_path

  # collect pm statistics from each function
  arg_values += get_pm_ip(get_pm_default_interface())
  arg_values += ["online"]
  arg_values += get_pm_cpu_info()
  arg_values += get_pm_mem_info()
  arg_values += get_pm_disk_info()
  arg_values += get_pm_network_info()
  arg_values += [capability]
  arg_values += [str(port)]

  # create conn to xen and kvm hypervisor
  xen_support = False
  qemu_support = False
  xen_domains_ID = []
  qemu_domains_ID = []
  conn_xen = None
  conn_qemu = None
  xen_domains_info = []
  qemu_domains_info = []

  try:
    conn_xen = libvirt.openReadOnly("xen:///")
  except:
    print "Can not connect to xend!"
  try:
    conn_qemu = libvirt.openReadOnly("qemu:///system")
  except:
    print "Can not connect to qemu!" 
  if conn_xen != None:
    xen_support = True
    if conn_xen.numOfDomains() > 1:
      xen_domains_ID = conn_xen.listDomainsID()
    else:
      print "No xen domain except domain 0!"
  if conn_qemu != None:
    qemu_support = True
    if conn_qemu.numOfDomains() > 0:
      qemu_domains_ID = conn_qemu.listDomainsID()
    else:
      print "No qemu domain!"

  # collect domain statistics
  if xen_support == True and conn_xen.numOfDomains() > 1:
    xen_domains_info = get_domains_info(conn_xen, "xen", xen_domains_ID[1:len(xen_domains_ID)])       
  if qemu_support == True and conn_qemu.numOfDomains() > 0:
    qemu_domains_info = get_domains_info(conn_qemu, "qemu", qemu_domains_ID)

  for i in xrange(13): # 13= len(arg_names)-15
    value = ""
    if xen_domains_info != []:
      for xen_domain_item in xen_domains_info:
        value += str(xen_domain_item[i]) + " "
    if qemu_domains_info != []:
      for qemu_domain_item in qemu_domains_info:
        value += str(qemu_domain_item[i]) + " "
    arg_values += [value]
    
  # all pair ping
  # 1. get_hosts_url from portal
  get_hosts_url = config_monitor['portal_url'] + '/vmcs/get_hosts/all'
  p = subprocess.Popen(['curl', '--cert', client_key, '--cacert', server_cert, get_hosts_url], stdout=subprocess.PIPE)
  p.wait()
  s = p.stdout.readline()
  target_dict = eval(s.replace('=>',':'))
  target_hostnames = target_dict.keys() # e.g. ['jan', 'feb', 'cnn.com']
  target_addresses = target_dict.values() # e.g. ['1.2.3.4', '2.3.4.5', '192.168.100.1']
  
  # 2. ping targets
  pkg_losts, rtts = ping_targets(target_addresses)
  app_dict = {}
  for i in target_hostnames:
    app_dict[i] = [pkg_losts[target_hostnames.index(i)], list(rtts[target_hostnames.index(i)])]
  
  arg_values += [repr(app_dict)]
  # all pair ping
  
  cmd = "curl -X PUT --cert " + client_key + " --cacert " + server_cert + " -d \""
  for i in xrange(len(arg_names)):
    cmd += "VirtualMachineContainer[" + arg_names[i] + "]=" + arg_values[i] + "&"
  cmd += "\" -H \"Accept: text/xml\" " + send_url
  os.system(cmd)
  print cmd      

# add by wty, all pair ping function
def ping_targets(targets, ping_count=5):
  create_sub_process = lambda x: subprocess.Popen(['ping', '-w', '1', '-c', '%s' % ping_count, x], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  processes = map(create_sub_process, targets)
  map(lambda x: x.wait(), processes)
  stdouts = map(lambda x: x.stdout.readlines(), processes)
  stderrs = map(lambda x: x.stderr.readlines(), processes)
  
  # 5 packets transmitted, 5 received, 0% packet loss, time 4001ms
  prog_pkg_lost = re.compile(r".*, ([0-9\.]+%) packet loss,.*")
  match = lambda x: prog_pkg_lost.match(x)
  
  def get_pkg_lost(stdout):
    m = map(match, stdout)
    mm = filter(None, m)
    if mm:	return mm[0].groups()[0]
    else: return ''
    
  pkg_losts = map(lambda x: get_pkg_lost(x), stdouts)
    
  # rtt min/avg/max/mdev = 0.117/0.140/0.159/0.021 ms
  prog_rtt = re.compile(r"rtt min/avg/max/mdev = ([0-9\.]*)/([0-9\.]*)/([0-9\.]*)/([0-9\.]*) ms")
  match = lambda x: prog_rtt.match(x)
  def get_rtt(stdout):
    m = map(match, stdout)
    mm = filter(None, m)
    if mm:	return mm[0].groups()
    else: return ['', '', '', '']
  rtts = map(lambda x: get_rtt(x), stdouts)
  
  return pkg_losts, rtts
# end of all pair ping function


# add by liqin, invoked by /usr/bin/ivic-monitor
def main():
  try:
    check_pid_file(pid_file)
  except:
    print 'another instance is already running, exit'
    return 
  fd = file(pid_file, "w")
  print >> fd, os.getpid()
  fd.close()
  do_main(sys.argv)
